package mygroup.test.hadooptest;

import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.wltea.analyzer.lucene.IKAnalyzer;

public class MyMap extends Mapper<Object,Text,Text,IntWritable>{
	
	private static final IntWritable one =  new IntWritable(1);  

    protected void map(Object key, Text value, Context context)  
            throws IOException, InterruptedException {  
    	//value="  <title>hadoop2.6.0版本搭建伪分布式环境 - stark_summer - ITeye技术网站</title>
        String line = value.toString();  
        String regex="[\u4e00-\u9fa5]+";// 中文
        String regex_="[a-zA-Z0-9]+"; // 字母
        if(isContainWanted(regex, line)){ //has zH
        	//  
        	for(String word:analyzer(line)){
        		if(! isContainWanted(regex_,word)){
        			context.write(new Text(word), one);  
        		} 
        	}
        	 
        }
        
    }  
    
    private  boolean isContainWanted(String regex,String  str) {
        Matcher matcher = Pattern.compile(regex).matcher(str);
        if(matcher.find()){
            System.out.println(matcher.group(0));
            return true;
        }
        return false;
    }

    private  List<String> analyzer(String line){
    	List l = new ArrayList<String>();
    	//构建IK分词器，使用smart分词模式
    			Analyzer analyzer = new IKAnalyzer(true);
    			
    			//获取Lucene的TokenStream对象
    		    TokenStream ts = null;
    			try {
    				ts = analyzer.tokenStream("myfield", new StringReader(line));
    				//获取词元位置属性
    			    OffsetAttribute  offset = ts.addAttribute(OffsetAttribute.class); 
    			    //获取词元文本属性
    			    CharTermAttribute term = ts.addAttribute(CharTermAttribute.class);
    			    //获取词元文本属性
    			    TypeAttribute type = ts.addAttribute(TypeAttribute.class);
    			    
    			    
    			    //重置TokenStream（重置StringReader）
    				ts.reset(); 
    				//迭代获取分词结果
    				while (ts.incrementToken()) {
    				  //System.out.println(offset.startOffset() + " - " + offset.endOffset() + " : " + term.toString() + " | " + type.type());
    					//System.out.println(term);
    					l.add(term);
    				}
    				//关闭TokenStream（关闭StringReader）
    				ts.end();   // Perform end-of-stream operations, e.g. set the final offset.

    			} catch (IOException e) {
    				e.printStackTrace();
    			} finally {
    				//释放TokenStream的所有资源
    				if(ts != null){
    			      try {
    					ts.close();
    			      } catch (IOException e) {
    					e.printStackTrace();
    			      }
    				}
    		    }
    			
    	
    	return l;
    }
}
